{
 "cells": [
  {
   "metadata": {},
   "cell_type": "raw",
   "source": "数据预处理\n"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-28T04:44:22.864140Z",
     "start_time": "2025-03-28T04:44:22.809800Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import numpy as np\n",
    "import struct\n",
    "import os\n",
    "\n",
    "\n",
    "def read_idx(data):\n",
    "    \"\"\"\n",
    "    读取 IDX 文件 (MNIST 数据格式).\n",
    "    \"\"\"\n",
    "    with open(data, 'rb') as f:\n",
    "        # 读取 Magic Number 和维度信息\n",
    "        # >ii 表示两个大端序(big-endian)的 4 字节整数\n",
    "        zero, data_type_code, num_dimensions = struct.unpack('>HBB', f.read(4))\n",
    "        # 根据数据类型代码确定 numpy 的 dtype\n",
    "        # 0x08: unsigned byte\n",
    "        if data_type_code == 0x08:\n",
    "            dtype = np.uint8\n",
    "        else:\n",
    "            # 可以根据需要添加对其他类型的支持\n",
    "            raise ValueError(f\"Unsupported data type code: {data_type_code}\")\n",
    "\n",
    "        # 读取每个维度的大小\n",
    "        # '>' + 'I' * num_dimensions 表示读取 num_dimensions 个大端序的 4 字节无符号整数\n",
    "        dimension_sizes = struct.unpack(\n",
    "            '>' + 'I' * num_dimensions, f.read(4 * num_dimensions))\n",
    "\n",
    "        # 计算数据总大小\n",
    "        total_size = np.prod(dimension_sizes)\n",
    "\n",
    "        # 读取数据部分\n",
    "        data = np.frombuffer(f.read(total_size), dtype=dtype)\n",
    "\n",
    "        # 将数据重塑为正确的维度\n",
    "        data = data.reshape(dimension_sizes)\n",
    "\n",
    "        return data\n",
    "\n",
    "\n",
    "data_dir = r'./data/'  # 使用原始字符串避免转义问题\n",
    "\n",
    "# 检查路径是否存在\n",
    "if not os.path.isdir(data_dir):\n",
    "    print(f\"错误：目录不存在 - {data_dir}\")\n",
    "else:\n",
    "    try:\n",
    "        # --- 加载训练数据 ---\n",
    "        train_images_path = os.path.join(data_dir, 'train-images.idx3-ubyte')\n",
    "        train_labels_path = os.path.join(data_dir, 'train-labels.idx1-ubyte')\n",
    "\n",
    "        if os.path.exists(train_images_path) and os.path.exists(train_labels_path):\n",
    "            print(\"正在加载训练数据...\")\n",
    "            X_train = read_idx(train_images_path)\n",
    "            y_train = read_idx(train_labels_path)\n",
    "            print(f\"训练图像形状: {X_train.shape}\")  # 应该是 (60000, 28, 28)\n",
    "            print(f\"训练标签形状: {y_train.shape}\")  # 应该是 (60000,)\n",
    "        else:\n",
    "            print(\"警告：训练数据文件未找到。\")\n",
    "\n",
    "        # --- 加载测试数据 ---\n",
    "        test_images_path = os.path.join(data_dir, 't10k-images.idx3-ubyte')\n",
    "        test_labels_path = os.path.join(data_dir, 't10k-labels.idx1-ubyte')\n",
    "\n",
    "        if os.path.exists(test_images_path) and os.path.exists(test_labels_path):\n",
    "            print(\"\\n正在加载测试数据...\")\n",
    "            X_test = read_idx(test_images_path)\n",
    "            y_test = read_idx(test_labels_path)\n",
    "            print(f\"测试图像形状: {X_test.shape}\")   # 应该是 (10000, 28, 28)\n",
    "            print(f\"测试标签形状: {y_test.shape}\")   # 应该是 (10000,)\n",
    "        else:\n",
    "            print(\"警告：测试数据文件未找到。\")\n",
    "\n",
    "        # 使用 X_train, y_train, X_test, y_test 进行后续处理 ---\n",
    "        # 例如，在 scikit-learn 中使用前，通常需要将图像数据展平：\n",
    "        if 'X_train' in locals():\n",
    "            X_train_flat = X_train.reshape(X_train.shape[0], -1)\n",
    "            print(f\"\\n展平后的训练图像形状: {X_train_flat.shape}\")  # (60000, 784)\n",
    "        if 'X_test' in locals():\n",
    "            X_test_flat = X_test.reshape(X_test.shape[0], -1)\n",
    "            print(f\"展平后的测试图像形状: {X_test_flat.shape}\")   # (10000, 784)\n",
    "\n",
    "        # 将它们包装成 scikit-learn 的 Bunch 对象\n",
    "        from sklearn.utils import Bunch\n",
    "        mnist_data = Bunch(\n",
    "            data=X_train_flat if 'X_train_flat' in locals() else None,\n",
    "            target=y_train if 'y_train' in locals() else None,\n",
    "            images=X_train if 'X_train' in locals() else None,\n",
    "            DESCR=\"手动加载的 MNIST 训练数据\",\n",
    "            data_test=X_test_flat if 'X_test_flat' in locals() else None,\n",
    "            target_test=y_test if 'y_test' in locals() else None,\n",
    "            images_test=X_test if 'X_test' in locals() else None\n",
    "        )\n",
    "        print(mnist_data.keys())\n",
    "\n",
    "    except FileNotFoundError as e:\n",
    "        print(f\"错误：文件未找到 - {e}\")\n",
    "    except Exception as e:\n",
    "        print(f\"发生错误: {e}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正在加载训练数据...\n",
      "训练图像形状: (60000, 28, 28)\n",
      "训练标签形状: (60000,)\n",
      "\n",
      "正在加载测试数据...\n",
      "测试图像形状: (10000, 28, 28)\n",
      "测试标签形状: (10000,)\n",
      "\n",
      "展平后的训练图像形状: (60000, 784)\n",
      "展平后的测试图像形状: (10000, 784)\n",
      "发生错误: No module named 'sklearn'\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "数据预处理使得数据标准化，统一化"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-28T04:46:19.558324Z",
     "start_time": "2025-03-28T04:46:19.534799Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X_train = None\n",
    "y_train = None\n",
    "X_test = None\n",
    "y_test = None\n",
    "\n",
    "try:\n",
    "    print(\"正在加载数据...\")\n",
    "    train_images_path = os.path.join(data_dir, 'train-images.idx3-ubyte')\n",
    "    train_labels_path = os.path.join(data_dir, 'train-labels.idx1-ubyte')\n",
    "    test_images_path = os.path.join(data_dir, 't10k-images.idx3-ubyte')\n",
    "    test_labels_path = os.path.join(data_dir, 't10k-labels.idx1-ubyte')\n",
    "\n",
    "    required_files = [train_images_path, train_labels_path,\n",
    "                      test_images_path, test_labels_path]\n",
    "    all_files_exist = True\n",
    "    for f_path in required_files:\n",
    "        if not os.path.exists(f_path):\n",
    "            print(f\"错误：文件未找到 - {f_path}\")\n",
    "            all_files_exist = False\n",
    "\n",
    "    if all_files_exist:\n",
    "        X_train = read_idx(train_images_path)\n",
    "        y_train = read_idx(train_labels_path)\n",
    "        X_test = read_idx(test_images_path)\n",
    "        y_test = read_idx(test_labels_path)\n",
    "        print(\"数据加载完成。\")\n",
    "        print(f\"训练图像形状: {X_train.shape}\")\n",
    "        print(f\"训练标签形状: {y_train.shape}\")\n",
    "        print(f\"测试图像形状: {X_test.shape}\")\n",
    "        print(f\"测试标签形状: {y_test.shape}\")\n",
    "    else:\n",
    "        print(\"数据加载失败，缺少文件。\")\n",
    "        exit()  # Exit if data loading failed\n",
    "\n",
    "except Exception as e:\n",
    "    print(f\"加载数据时发生错误: {e}\")\n",
    "    exit()"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正在加载数据...\n",
      "加载数据时发生错误: name 'os' is not defined\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-28T04:41:10.251738Z",
     "start_time": "2025-03-28T04:41:08.918837Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "print(torch.cuda.is_available())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "execution_count": 1
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DataAnalysis",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
